#include <iostream>
#include <string>
#include <vector>
#include <boost/filesystem.hpp>
#include "Util.hpp"

static const std::string file_path = "./data/input";
static const std::string raw_path = "./data/raw_html/raw.txt";

struct DocInfo
{
    std::string _title;
    std::string _content;
    std::string _url;
};

// 参数风格：const &输入型参数，&输入输出型参数。*输出型参数
static bool GetAllPaths(const std::string &file_path, std::vector<std::string> *html_paths);
static bool ParseData(const std::string &page_content, std::vector<DocInfo> *docs, const std::string &path);
static bool SaveHtml(const std::vector<DocInfo> &docs, const std::string &raw_path);

int main()
{
    std::vector<std::string> html_paths;
    // 1.读取所有html网页的路径，保存起来
    if (!GetAllPaths(file_path, &html_paths))
    {
        std::cerr << "get paths error" << std::endl;
        return 1;
    }
    // html_paths.push_back("./data/input/test.html");
    std::cout << html_paths.size() << std::endl;

    // 2.遍历html_paths，
    std::vector<DocInfo> docs;
    for (const std::string &path : html_paths)
    {
        std::string page_content;
        // 根据路径，读取对应html全文内容
        if (!ns_util::FileUtil::ReadOnePage(path, &page_content))
        {
            std::cerr << "ReadOnePage error" << std::endl;
            return 2;
        }
        // // for debug
        // std::cout << page_content << std::endl;
        // 3.数据清洗：取标题，去标签，拼接url
        if (!ParseData(page_content, &docs, path))
        {
            std::cerr << "ParseData error" << std::endl;
            return 3;
        }
    }
    // 4.将清洗好的结构化数据写到文件中
    if (!SaveHtml(docs, raw_path))
    {
        std::cerr << "SaveHtml error" << std::endl;
        return 4;
    }

    return 0;
}

// 1.读取所有html网页的路径，保存起来
static bool GetAllPaths(const std::string &file_path, std::vector<std::string> *html_paths)
{
    namespace fs = boost::filesystem;
    fs::recursive_directory_iterator end;
    for (fs::recursive_directory_iterator cur(file_path); cur != end; ++cur)
    {
        // 不是普通文件，忽略
        if (!fs::is_regular_file(*cur))
            continue;
        // 文件后缀不是.html。忽略
        if (fs::extension(*cur) != ".html")
            continue;
        // 记录到html_paths中
        html_paths->push_back(cur->path().string());
    }
    return true;
}

static bool GetTitle(const std::string &page_content, std::string *title)
{
    std::string titleString("<title>");
    size_t begin = page_content.find(titleString);
    if (begin == std::string::npos)
        return false;
    size_t end = page_content.find("</title>");
    if (end == std::string::npos)
        return false;
    // 起始位置：begin + title标签长度
    // 截取长度：end-begin-title标签长度
    *title = page_content.substr(begin + titleString.size(), end - begin - titleString.size());
    return true;
}

static void RemoveLabel(const std::string &page_content, std::string *content)
{
    enum status
    {
        LABEL,
        CONTENT
    };
    status cur = LABEL;
    for (char c : page_content)
    {
        switch (cur)
        {
        case LABEL:
        {
            if (c == '>')
                cur = CONTENT;
            break;
        }
        case CONTENT:
        {
            if (c == '<')
                cur = LABEL;
            else
            {
                // ‘\n’是用于分隔文件中两个DocInfo的
                if (c == '\n')
                    c = ' ';
                content->push_back(c);
            }
            break;
        }
        default:
            break;
        }
    }
}

static void GetUrl(const std::string &page_content, std::string *url, const std::string &path)
{
    std::string url_head = "https://www.boost.org/doc/libs/1_89_0/doc/html";
    // path: ./data/input/xxx/xxx.html -> url结尾：/xxx/xxx.html
    std::string url_tail = path.substr(file_path.size());
    *url = url_head + url_tail;
}

// 3.数据清洗：取标题，去标签，拼接url
static bool ParseData(const std::string &page_content, std::vector<DocInfo> *docs, const std::string &path)
{
    DocInfo doc;
    // 1.取标题
    if (!GetTitle(page_content, &doc._title))
        return false;
    // 2.去标签
    RemoveLabel(page_content, &doc._content);
    // 3.拼接url
    GetUrl(page_content, &doc._url, path);

    // 插入到docs中
    docs->push_back(std::move(doc));
    return true;
}

// 4.将清洗好的结构化数据写到文件中
static bool SaveHtml(const std::vector<DocInfo> &docs, const std::string &raw_path)
{
    std::ofstream ofs(raw_path, std::ios::trunc | std::ios::out); // 清空写
    if (!ofs.is_open())
        return false;
    for (const DocInfo &doc : docs)
    {
        // title\3content\3url\n
        ofs << doc._title << '\3' << doc._content << '\3' << doc._url << '\n';
        ofs.flush();
    }
    ofs.close();
    return true;
}